{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.nn import init\n",
    "import math\n",
    "import os\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ChannelAttention(nn.Module):\n",
    "    def __init__(self, in_planes, ratio=16):\n",
    "        super(ChannelAttention, self).__init__()\n",
    "        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n",
    "        self.max_pool = nn.AdaptiveMaxPool2d(1)\n",
    "        self.fc1   = nn.Conv2d(in_planes, in_planes // 16, 1, bias=False)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.fc2   = nn.Conv2d(in_planes // 16, in_planes, 1, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x))))\n",
    "        max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x))))\n",
    "        out = avg_out + max_out\t\t# 这里并没有使用到论文中的shared MLP, 而是直接相加了\n",
    "        return self.sigmoid(out)\n",
    "\n",
    "\n",
    "class SpatialAttention(nn.Module):\n",
    "    def __init__(self, kernel_size=7):\n",
    "        super(SpatialAttention, self).__init__()\n",
    "        assert kernel_size in (3, 7), 'kernel size must be 3 or 7'\n",
    "        padding = 3 if kernel_size == 7 else 1\n",
    "        self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x):\n",
    "        avg_out = torch.mean(x, dim=1, keepdim=True)\n",
    "        max_out, _ = torch.max(x, dim=1, keepdim=True)\n",
    "        x = torch.cat([avg_out, max_out], dim=1)\n",
    "        x = self.conv1(x)\n",
    "        return self.sigmoid(x)\n",
    "def conv3x3(in_planes, out_planes, stride=1):\n",
    "    \"3x3 convolution with padding\"\n",
    "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
    "                     padding=1, bias=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SEModule(nn.Module):\n",
    "    expansion = 1\n",
    "\n",
    "    def __init__(self, inplanes, stride=1, downsample=None):\n",
    "        super(SEModule, self).__init__()\n",
    "        self.conv1 = conv3x3(inplanes, inplanes, stride)\n",
    "        self.bn1 = nn.BatchNorm2d(inplanes)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.conv2 = conv3x3(inplanes, inplanes)\n",
    "        self.bn2 = nn.BatchNorm2d(inplanes)\n",
    "\n",
    "        self.ca = ChannelAttention(inplanes)\n",
    "        self.sa = SpatialAttention()\n",
    "\n",
    "        self.downsample = downsample\n",
    "        self.stride = stride\n",
    "\n",
    "    def forward(self, x):\n",
    "        residual = x\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.bn1(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        out = self.conv2(out)\n",
    "        out = self.bn2(out)\n",
    "\n",
    "        out = self.ca(out) * out\n",
    "       \n",
    "        out += residual\n",
    "      \n",
    "#         print(out.size())\n",
    "        out = self.sa(out) * out\n",
    "#         print(out.size())\n",
    "        if self.downsample is not None:\n",
    "            residual = self.downsample(x)\n",
    "\n",
    "        out += residual\n",
    "#         print(out.size())\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conv31(ip,op,stride=1):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "def conv32(ip,op,stride=2):\n",
    "    return nn.Conv2d(ip,op,kernel_size=3,stride=stride,padding=1,bias=False)\n",
    "class Upsample(nn.Module):\n",
    "    def __init__(self,ip,op):\n",
    "        super(Upsample,self).__init__()\n",
    "        self.upsample=nn.Sequential(\n",
    "            conv31(ip,op,1),\n",
    "            nn.Upsample(scale_factor=2,mode='nearest')\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x=self.upsample(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def backward_hook(module,grad_in,grad_out):\n",
    "#     grad_block.append(grad_out[0].detach())\n",
    "\n",
    "# def farward_hook(module,input,output):\n",
    "#     fmap_block.append(output)\n",
    "\n",
    "    \n",
    "# def backward2_hook(module,grad_in,grad_out):\n",
    "#     grad_block2.append(grad_out[0].detach())\n",
    "# def backward3_hook(module,grad_in,grad_out):\n",
    "#     grad_block3.append(grad_out[0].detach())\n",
    "# fmap_block=[]\n",
    "# grad_block=[]\n",
    "# grad_block2=[]\n",
    "# grad_block3=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Hswish(x,inplace=True):\n",
    "    return x * F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "def Hsigmoid(x,inplace=True):\n",
    "    return F.relu6(x + 3., inplace=inplace) / 6.\n",
    "\n",
    "\n",
    "class Bottleneck(nn.Module):\n",
    "    def __init__(self,in_channels,out_channels,kernel_size,exp_channels,stride,se='True',nl='HS'):\n",
    "        super(Bottleneck, self).__init__()\n",
    "        padding = (kernel_size - 1) // 2\n",
    "        if nl == 'RE':\n",
    "            self.nlin_layer = F.relu6\n",
    "        elif nl == 'HS':\n",
    "            self.nlin_layer = Hswish\n",
    "        self.stride=stride\n",
    "        if se:\n",
    "            self.se=SEModule(exp_channels)\n",
    "        else:\n",
    "            self.se=None\n",
    "        self.conv1=nn.Conv2d(in_channels,exp_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(exp_channels)\n",
    "        self.conv2=nn.Conv2d(exp_channels,exp_channels,kernel_size=kernel_size,stride=stride,\n",
    "                             padding=padding,groups=exp_channels,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(exp_channels)\n",
    "        self.conv3=nn.Conv2d(exp_channels,out_channels,kernel_size=1,stride=1,padding=0,bias=False)\n",
    "        self.bn3=nn.BatchNorm2d(out_channels)\n",
    "        # ÏÈ³õÊ¼»¯Ò»¸ö¿ÕÐòÁÐ£¬Ö®ºó¸ÄÔìÆä³ÉÎª²Ð²îÁ´½Ó\n",
    "        self.shortcut = nn.Sequential()\n",
    "        # Ö»ÓÐ²½³¤Îª1ÇÒÊäÈëÊä³öÍ¨µÀ²»ÏàÍ¬Ê±²Å²ÉÓÃÌøÔ¾Á¬½Ó(ÏëÒ»ÏÂÌøÔ¾Á´½ÓµÄ¹ý³Ì£¬ÊäÈëÊä³öÍ¨µÀÏàÍ¬Õâ¸öÌøÔ¾Á¬½Ó¾ÍÃ»ÒâÒåÁË)\n",
    "        if stride == 1 and in_channels != out_channels:\n",
    "            self.shortcut = nn.Sequential(\n",
    "                # ÏÂÃæµÄ²Ù×÷¾í»ý²»¸Ä±ä³ß´çºÍÍ¨µÀÊý\n",
    "                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=1, padding=0, bias=False),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "\n",
    "    def forward(self,x):\n",
    "        out=self.nlin_layer(self.bn1(self.conv1(x)))\n",
    "        if self.se is not None:\n",
    "            out=self.bn2(self.conv2(out))\n",
    "            out=self.nlin_layer(self.se(out))\n",
    "        else:\n",
    "            out = self.nlin_layer(self.bn2(self.conv2(out)))\n",
    "        out = self.bn3(self.conv3(out))\n",
    "        out = out + self.shortcut(x) if self.stride == 1 else out\n",
    "        return out\n",
    "\n",
    "\n",
    "class MobileNetV3_large(nn.Module):\n",
    "    # (out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "    cfg=[\n",
    "        (16,3,16,1,False,'RE'),\n",
    "        (24,3,64,2,False,'RE'),\n",
    "        (24,3,72,1,False,'RE'),\n",
    "        (40,5,72,2,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (40,5,120,1,True,'RE'),\n",
    "        (80,3,240,2,False,'HS'),\n",
    "        (80,3,200,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (80,3,184,1,False,'HS'),\n",
    "        (112,3,480,1,True,'HS'),\n",
    "        (112,3,672,1,True,'HS'),\n",
    "        (160,5,672,2,True,'HS'),\n",
    "        (160,5,960,1,True,'HS'),\n",
    "        (160,5,960,1,True,'HS')\n",
    "    ]\n",
    "    def __init__(self,num_classes=5):\n",
    "        super(MobileNetV3_large,self).__init__()\n",
    "        self.conv1=nn.Conv2d(3,16,3,2,padding=1,bias=False)\n",
    "        self.bn1=nn.BatchNorm2d(16)\n",
    "        # ¸ù¾ÝcfgÊý×é×Ô¶¯Éú³ÉËùÓÐµÄBottleneck²ã\n",
    "        self.layers = self._make_layers(in_channels=16)\n",
    "        self.conv2=nn.Conv2d(160,960,1,stride=1,bias=False)\n",
    "        self.bn2=nn.BatchNorm2d(960)\n",
    "        # ¾í»ýºó²»¸úBN£¬¾ÍÓ¦¸Ã°ÑbiasÉèÖÃÎªTrue\n",
    "        self.conv3=nn.Conv2d(960,1280,1,1,padding=0,bias=True)\n",
    "        self.conv4=nn.Conv2d(1280,num_classes,1,stride=1,padding=0,bias=True)\n",
    "        self.up1=Upsample(160,112)\n",
    "        self.up2=Upsample(112,40)\n",
    "        self.c1=nn.Conv2d(224,112,kernel_size=3,stride=1,padding=1)\n",
    "        self.c2=nn.Conv2d(80,40,1,1)\n",
    "        self.c3=nn.Conv2d(40,112,kernel_size=3,stride=2,padding=1)\n",
    "        self.b4=nn.BatchNorm2d(112)\n",
    "        \n",
    "        self.c6=nn.Conv2d(112,160,kernel_size=3,stride=2,padding=1)\n",
    "        self.b7=nn.BatchNorm2d(160)\n",
    "        self.h1=nn.Conv2d(80,40,1,1)\n",
    "        self.h2=nn.Conv2d(224,112,1,1)\n",
    "        self.h3=nn.Conv2d(320,160,1,1)\n",
    "        \n",
    "#         self.w=nn.Parameter(torch.FloatTensor(1),requires_grad=True)\n",
    "#         self.w=torch.FloatTensor(1)\n",
    "        self.weights_try=0.1\n",
    "        self.weights2_try=0.2\n",
    "        self.weights3_try=0.7\n",
    "        \n",
    "#         self.w.data.fill_(1)\n",
    "#         self.weights2_try.data.fill_(1)\n",
    "#         self.weights3_try.data.fill_(1)\n",
    "        \n",
    "    def _make_layers(self,in_channels):\n",
    "        layers=[]\n",
    "        for out_channels,kernel_size,exp_channels,stride,se,nl in self.cfg:\n",
    "            layers.append(\n",
    "                Bottleneck(in_channels,out_channels,kernel_size,exp_channels,stride,se,nl)\n",
    "            )\n",
    "            in_channels=out_channels\n",
    "        return nn.Sequential(*layers)\n",
    "                \n",
    "\n",
    "    def forward(self,x):\n",
    "        out=Hswish(self.bn1(self.conv1(x)))\n",
    "        ##############################################\n",
    "        out1=self.layers[:6](out)\n",
    "        out2=self.layers[6:12](out1)\n",
    "        out3=self.layers[12:16](out2)\n",
    "        \n",
    "        out3x=self.weights3_try*out3\n",
    "        out1x=self.weights_try*out1\n",
    "        out2x=self.weights2_try*out2\n",
    "        \n",
    "        out3=torch.cat([out3,out3x],1)\n",
    "        out3=self.h3(out3)\n",
    "        out2=torch.cat([out2,out2x],1)\n",
    "        out2=self.h2(out2)\n",
    "        out1=torch.cat([out1,out1x],1)\n",
    "        out1=self.h1(out1)\n",
    "        \n",
    "        p5=self.up1(out3)\n",
    "        p4=torch.cat([p5,out2],1)\n",
    "#         print(p4.size()) #14*14*224\n",
    "        p4=self.c1(p4)\n",
    "        p4=self.up2(p4)\n",
    "#         print(p4.size())\n",
    "        p3=torch.cat([p4,out1],1)\n",
    "        p3=self.c2(p3)\n",
    "        p3=self.c3(p3)\n",
    "        p3=self.b4(p3)\n",
    "        p3=Hswish(p3)\n",
    "        \n",
    "        p3=self.c6(p3)\n",
    "        p3=self.b7(p3)\n",
    "        p3=Hswish(p3)\n",
    "        \n",
    "        out=Hswish(self.bn2(self.conv2(p3)))\n",
    "        out=F.avg_pool2d(out,7)\n",
    "        out=Hswish(self.conv3(out))\n",
    "        out=self.conv4(out)\n",
    "        \n",
    "        a,b=out.size(0),out.size(1)\n",
    "        out=out.view(a,b)\n",
    "#         print(self.weights_try)\n",
    "#         return out,out1,out2,out3\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=MobileNetV3_large()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x=torch.randn(2,3,224,224)\n",
    "# y=model(x)\n",
    "# print(y.size())\n",
    "# print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision\n",
    "from torch.autograd import Variable\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "from PIL import Image\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "random.seed(1)\n",
    "\n",
    "class flowerDataset(Dataset):\n",
    "    # 自定义Dataset类，必须继承Dataset并重写__init__和__getitem__函数\n",
    "    def __init__(self, data_dir, transform=None):\n",
    "        \"\"\"\n",
    "            花朵分类任务的Dataset\n",
    "            :param data_dir: str, 数据集所在路径\n",
    "            :param transform: torch.transform，数据预处理，默认不进行预处理\n",
    "        \"\"\"\n",
    "        # data_info存储所有图片路径和标签（元组的列表），在DataLoader中通过index读取样本\n",
    "        self.data_info = self.get_img_info(data_dir)\n",
    "        self.transform = transform\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        path_img, label = self.data_info[index]\n",
    "        # 打开图片，默认为PIL，需要转成RGB\n",
    "        img = Image.open(path_img).convert('RGB')\n",
    "        # 如果预处理的条件不为空，应该进行预处理操作\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        return img, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data_info)\n",
    "\n",
    "    # 自定义方法，用于返回所有图片的路径以及标签\n",
    "    @staticmethod\n",
    "    def get_img_info(data_dir):\n",
    "        data_info = list()\n",
    "        for root, dirs, _ in os.walk(data_dir):\n",
    "            # 遍历类别\n",
    "            for sub_dir in dirs:\n",
    "                # listdir为列出文件夹下所有文件和文件夹名\n",
    "                img_names = os.listdir(os.path.join(root, sub_dir))\n",
    "                # 过滤出所有后缀名为jpg的文件名（那当然也就把文件夹过滤掉了）\n",
    "                img_names = list(filter(lambda x: x.endswith('.jpeg'), img_names))\n",
    "\n",
    "                # 遍历图片\n",
    "                for i in range(len(img_names)):\n",
    "                    img_name = img_names[i]\n",
    "                    path_img = os.path.join(root, sub_dir, img_name)\n",
    "                    # 在该任务中，文件夹名等于标签名\n",
    "                    label = sub_dir\n",
    "                    data_info.append((path_img, int(label)))\n",
    "        return data_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#宏定义一些数据，如epoch数，batchsize等\n",
    "MAX_EPOCH=45\n",
    "BATCH_SIZE=128\n",
    "# LR=0.0001\n",
    "LR=0.001\n",
    "log_interval=30\n",
    "val_interval=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 1/5 数据 ============================\n",
    "# split_dir=os.path.join(\"MobileNetV3-master\",\".\",\"data\",\"splitData\")\n",
    "split_dir=r'/home/fanrz/Desktop/torchmob/'\n",
    "train_dir=r'/home/fanrz/Desktop/testlarge/train/'\n",
    "valid_dir=os.path.join(split_dir,\"val\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对训练集所需要做的预处理\n",
    "train_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对验证集所需要做的预处理\n",
    "valid_transform=transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建MyDataset实例\n",
    "train_data=flowerDataset(data_dir=train_dir,transform=train_transform)\n",
    "valid_data=flowerDataset(data_dir=valid_dir,transform=valid_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51033\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data.data_info))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建DataLoader\n",
    "# 训练集数据最好打乱\n",
    "# DataLoader的实质就是把数据集加上一个索引号，再返回\n",
    "train_loader=DataLoader(dataset=train_data,\n",
    "                        batch_size=BATCH_SIZE,\n",
    "                        shuffle=True,\n",
    "                        drop_last=True\n",
    "                       )\n",
    "valid_loader=DataLoader(dataset=valid_data,batch_size=BATCH_SIZE,drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 2/5 模型 ============================\n",
    "if torch.cuda.is_available():\n",
    "    model=nn.DataParallel(model)\n",
    "    model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ============================ step 3/5 损失函数 ============================\n",
    "criterion=nn.CrossEntropyLoss()\n",
    "# ============================ step 4/5 优化器 ============================\n",
    "optimizer=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器\n",
    "\n",
    "# optimizer2=optim.Adam(model.parameters(),lr=LR, betas=(0.9, 0.99))# 选择优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gs(output,label):\n",
    "    s=0.0\n",
    "    for i,n in enumerate(label):\n",
    "        s+=output[i][n]\n",
    "    return s/(i+1)\n",
    "def gw(weights):\n",
    "    s=0\n",
    "    for i in weights:\n",
    "        if(i>0):\n",
    "            s+=i\n",
    "    return s\n",
    "def cw(grad_block,batch):\n",
    "    count_weight_list=[]\n",
    "    i=0\n",
    "    while(i<batch):\n",
    "        grads_val=grad_block[0][i].cpu().data.numpy().squeeze()\n",
    "        grads=grads_val.reshape(grads_val.shape[0],-1)\n",
    "        weights=np.mean(grads,axis=1)\n",
    "        w=gw(weights)\n",
    "        count_weight_list.append(w)\n",
    "        grads=0\n",
    "        grads_val=0\n",
    "        i+=1\n",
    "    return np.mean(count_weight_list)\n",
    "def count_final(grad_block,grad_block2,grad_block3,batch_size):\n",
    "    score_last=cw(grad_block,batch_size)\n",
    "    score_mid=cw(grad_block2,batch_size)\n",
    "    score_first=cw(grad_block3,batch_size)\n",
    "    w_last=score_last/(score_last+score_first+score_mid)\n",
    "    w_mid=score_mid/(score_last+score_first+score_mid)\n",
    "    w_first=score_first/(score_last+score_first+score_mid)\n",
    "    return w_first,w_mid,w_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/torch/nn/modules/upsampling.py:129: UserWarning: nn.Upsample is deprecated. Use nn.functional.interpolate instead.\n",
      "  warnings.warn(\"nn.{} is deprecated. Use nn.functional.interpolate instead.\".format(self.name))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:1,loss:1.4315\n",
      "0.35703125\n",
      "epoch:1,loss:1.5041\n",
      "0.37356770833333336\n",
      "epoch:1,loss:1.3590\n",
      "0.3853298611111111\n",
      "epoch:1,loss:1.4040\n",
      "0.39837239583333334\n",
      "epoch:1,loss:1.3092\n",
      "0.408125\n",
      "epoch:1,loss:1.3293\n",
      "0.4133246527777778\n",
      "epoch:1,loss:1.3007\n",
      "0.42176339285714287\n",
      "epoch:1,loss:1.2942\n",
      "0.42939453125\n",
      "epoch:1,loss:1.2489\n",
      "0.43712384259259257\n",
      "epoch:1,loss:1.2047\n",
      "0.4435677083333333\n",
      "epoch:1,loss:1.2543\n",
      "0.4491003787878788\n",
      "epoch:1,loss:1.1532\n",
      "0.45405815972222224\n",
      "epoch:1,loss:1.1610\n",
      "0.4584735576923077\n",
      "============================================\n",
      "准确率由： 0.0 上升至： 0.45954381281407036 已更新并保存权值为weights/best.pkl\n",
      "第1个epoch的识别准确率为：45%\n",
      "epoch:2,loss:1.1229\n",
      "0.5330729166666667\n",
      "epoch:2,loss:1.0991\n",
      "0.5307291666666667\n",
      "epoch:2,loss:1.0922\n",
      "0.5360243055555556\n",
      "epoch:2,loss:0.9826\n",
      "0.53515625\n",
      "epoch:2,loss:1.0886\n",
      "0.5375\n",
      "epoch:2,loss:1.0036\n",
      "0.5402777777777777\n",
      "epoch:2,loss:1.0138\n",
      "0.5436011904761905\n",
      "epoch:2,loss:0.9707\n",
      "0.5462239583333334\n",
      "epoch:2,loss:1.0958\n",
      "0.5482638888888889\n",
      "epoch:2,loss:0.9347\n",
      "0.5497135416666666\n",
      "epoch:2,loss:0.9286\n",
      "0.5513020833333333\n",
      "epoch:2,loss:1.0599\n",
      "0.5540581597222223\n",
      "epoch:2,loss:0.8616\n",
      "0.5550881410256411\n",
      "============================================\n",
      "准确率由： 0.45954381281407036 上升至： 0.5555904522613065 已更新并保存权值为weights/best.pkl\n",
      "第2个epoch的识别准确率为：55%\n",
      "epoch:3,loss:0.9277\n",
      "0.6010416666666667\n",
      "epoch:3,loss:0.8414\n",
      "0.5903645833333333\n",
      "epoch:3,loss:0.8933\n",
      "0.5934895833333333\n",
      "epoch:3,loss:0.8776\n",
      "0.5981770833333333\n",
      "epoch:3,loss:0.8594\n",
      "0.59796875\n",
      "epoch:3,loss:0.8012\n",
      "0.5977430555555555\n",
      "epoch:3,loss:0.8894\n",
      "0.6009300595238095\n",
      "epoch:3,loss:0.9040\n",
      "0.603515625\n",
      "epoch:3,loss:0.7457\n",
      "0.6060763888888889\n",
      "epoch:3,loss:0.7631\n",
      "0.6084635416666667\n",
      "epoch:3,loss:0.8507\n",
      "0.6107717803030303\n",
      "epoch:3,loss:0.7535\n",
      "0.6127170138888889\n",
      "epoch:3,loss:0.8611\n",
      "0.6149639423076924\n",
      "============================================\n",
      "准确率由： 0.5555904522613065 上升至： 0.6157152952261307 已更新并保存权值为weights/best.pkl\n",
      "第3个epoch的识别准确率为：61%\n",
      "epoch:4,loss:0.6433\n",
      "0.6731770833333334\n",
      "epoch:4,loss:0.6537\n",
      "0.6759114583333333\n",
      "epoch:4,loss:0.8645\n",
      "0.6734375\n",
      "epoch:4,loss:0.7372\n",
      "0.6705729166666666\n",
      "epoch:4,loss:0.7916\n",
      "0.67265625\n",
      "epoch:4,loss:0.7155\n",
      "0.6728732638888889\n",
      "epoch:4,loss:0.7300\n",
      "0.6737723214285715\n",
      "epoch:4,loss:0.7199\n",
      "0.67392578125\n",
      "epoch:4,loss:0.6144\n",
      "0.6750289351851851\n",
      "epoch:4,loss:0.6532\n",
      "0.676328125\n",
      "epoch:4,loss:0.6302\n",
      "0.6785037878787878\n",
      "epoch:4,loss:0.6454\n",
      "0.6799913194444445\n",
      "epoch:4,loss:0.6561\n",
      "0.6821514423076923\n",
      "============================================\n",
      "准确率由： 0.6157152952261307 上升至： 0.6821019158291457 已更新并保存权值为weights/best.pkl\n",
      "第4个epoch的识别准确率为：68%\n",
      "epoch:5,loss:0.6572\n",
      "0.7369791666666666\n",
      "epoch:5,loss:0.7638\n",
      "0.7328125\n",
      "epoch:5,loss:0.6054\n",
      "0.7298611111111111\n",
      "epoch:5,loss:0.6513\n",
      "0.7303385416666667\n",
      "epoch:5,loss:0.5602\n",
      "0.7324479166666666\n",
      "epoch:5,loss:0.5581\n",
      "0.7302951388888889\n",
      "epoch:5,loss:0.6725\n",
      "0.7313988095238095\n",
      "epoch:5,loss:0.4519\n",
      "0.7343424479166667\n",
      "epoch:5,loss:0.5241\n",
      "0.7347511574074074\n",
      "epoch:5,loss:0.5068\n",
      "0.73484375\n",
      "epoch:5,loss:0.5092\n",
      "0.737405303030303\n",
      "epoch:5,loss:0.5212\n",
      "0.7387152777777778\n",
      "epoch:5,loss:0.5309\n",
      "0.739042467948718\n",
      "============================================\n",
      "准确率由： 0.6821019158291457 上升至： 0.7392234610552764 已更新并保存权值为weights/best.pkl\n",
      "第5个epoch的识别准确率为：73%\n",
      "epoch:6,loss:0.5576\n",
      "0.77421875\n",
      "epoch:6,loss:0.4396\n",
      "0.774609375\n",
      "epoch:6,loss:0.5719\n",
      "0.7737847222222223\n",
      "epoch:6,loss:0.4750\n",
      "0.7743489583333333\n",
      "epoch:6,loss:0.5025\n",
      "0.7721875\n",
      "epoch:6,loss:0.5571\n",
      "0.7713107638888889\n",
      "epoch:6,loss:0.4841\n",
      "0.7712797619047619\n",
      "epoch:6,loss:0.4640\n",
      "0.7720703125\n",
      "epoch:6,loss:0.5425\n",
      "0.7706018518518518\n",
      "epoch:6,loss:0.5157\n",
      "0.770703125\n",
      "epoch:6,loss:0.4406\n",
      "0.7713304924242425\n",
      "epoch:6,loss:0.4755\n",
      "0.7709201388888889\n",
      "epoch:6,loss:0.4281\n",
      "0.7731370192307693\n",
      "============================================\n",
      "准确率由： 0.7392234610552764 上升至： 0.7732215766331658 已更新并保存权值为weights/best.pkl\n",
      "第6个epoch的识别准确率为：77%\n",
      "epoch:7,loss:0.4018\n",
      "0.79921875\n",
      "epoch:7,loss:0.5469\n",
      "0.808203125\n",
      "epoch:7,loss:0.4112\n",
      "0.8045138888888889\n",
      "epoch:7,loss:0.3973\n",
      "0.8014322916666666\n",
      "epoch:7,loss:0.3474\n",
      "0.8019791666666667\n",
      "epoch:7,loss:0.4695\n",
      "0.8013020833333333\n",
      "epoch:7,loss:0.4270\n",
      "0.8027529761904761\n",
      "epoch:7,loss:0.5272\n",
      "0.8012369791666667\n",
      "epoch:7,loss:0.5861\n",
      "0.8011284722222223\n",
      "epoch:7,loss:0.4630\n",
      "0.7994791666666666\n",
      "epoch:7,loss:0.4560\n",
      "0.7987926136363637\n",
      "epoch:7,loss:0.5153\n",
      "0.7989800347222222\n",
      "epoch:7,loss:0.4313\n",
      "0.8002804487179487\n",
      "============================================\n",
      "准确率由： 0.7732215766331658 上升至： 0.8001727386934674 已更新并保存权值为weights/best.pkl\n",
      "第7个epoch的识别准确率为：80%\n",
      "epoch:8,loss:0.3501\n",
      "0.8270833333333333\n",
      "epoch:8,loss:0.3940\n",
      "0.8346354166666666\n",
      "epoch:8,loss:0.4043\n",
      "0.8293402777777777\n",
      "epoch:8,loss:0.3813\n",
      "0.829296875\n",
      "epoch:8,loss:0.4069\n",
      "0.8281770833333333\n",
      "epoch:8,loss:0.3460\n",
      "0.8263020833333333\n",
      "epoch:8,loss:0.3664\n",
      "0.8275669642857143\n",
      "epoch:8,loss:0.4473\n",
      "0.8261393229166667\n",
      "epoch:8,loss:0.2770\n",
      "0.8254050925925925\n",
      "epoch:8,loss:0.3905\n",
      "0.824765625\n",
      "epoch:8,loss:0.3218\n",
      "0.8245265151515152\n",
      "epoch:8,loss:0.3473\n",
      "0.8243055555555555\n",
      "epoch:8,loss:0.4747\n",
      "0.8243990384615385\n",
      "============================================\n",
      "准确率由： 0.8001727386934674 上升至： 0.8241598618090452 已更新并保存权值为weights/best.pkl\n",
      "第8个epoch的识别准确率为：82%\n",
      "epoch:9,loss:0.3117\n",
      "0.8502604166666666\n",
      "epoch:9,loss:0.3045\n",
      "0.8565104166666667\n",
      "epoch:9,loss:0.3100\n",
      "0.8547743055555556\n",
      "epoch:9,loss:0.3482\n",
      "0.854296875\n",
      "epoch:9,loss:0.2614\n",
      "0.8550520833333334\n",
      "epoch:9,loss:0.3158\n",
      "0.8544704861111111\n",
      "epoch:9,loss:0.3155\n",
      "0.8534970238095239\n",
      "epoch:9,loss:0.2324\n",
      "0.85341796875\n",
      "epoch:9,loss:0.3315\n",
      "0.8537905092592593\n",
      "epoch:9,loss:0.2433\n",
      "0.8528125\n",
      "epoch:9,loss:0.2809\n",
      "0.8537642045454545\n",
      "epoch:9,loss:0.4279\n",
      "0.854296875\n",
      "epoch:9,loss:0.2991\n",
      "0.8535857371794872\n",
      "============================================\n",
      "准确率由： 0.8241598618090452 上升至： 0.8530935929648241 已更新并保存权值为weights/best.pkl\n",
      "第9个epoch的识别准确率为：85%\n",
      "epoch:10,loss:0.3347\n",
      "0.88515625\n",
      "epoch:10,loss:0.2122\n",
      "0.8819010416666667\n",
      "epoch:10,loss:0.2783\n",
      "0.8777777777777778\n",
      "epoch:10,loss:0.3279\n",
      "0.8785807291666666\n",
      "epoch:10,loss:0.3162\n",
      "0.87953125\n",
      "epoch:10,loss:0.2715\n",
      "0.8790798611111111\n",
      "epoch:10,loss:0.2182\n",
      "0.8790178571428572\n",
      "epoch:10,loss:0.3168\n",
      "0.8771158854166666\n",
      "epoch:10,loss:0.2123\n",
      "0.8759837962962963\n",
      "epoch:10,loss:0.2886\n",
      "0.8761458333333333\n",
      "epoch:10,loss:0.2937\n",
      "0.875\n",
      "epoch:10,loss:0.2810\n",
      "0.8744791666666667\n",
      "epoch:10,loss:0.1625\n",
      "0.8747796474358974\n",
      "============================================\n",
      "准确率由： 0.8530935929648241 上升至： 0.8748625942211056 已更新并保存权值为weights/best.pkl\n",
      "第10个epoch的识别准确率为：87%\n",
      "epoch:11,loss:0.2542\n",
      "0.8997395833333334\n",
      "epoch:11,loss:0.2232\n",
      "0.897265625\n",
      "epoch:11,loss:0.2915\n",
      "0.89609375\n",
      "epoch:11,loss:0.3011\n",
      "0.8984375\n",
      "epoch:11,loss:0.2454\n",
      "0.8954166666666666\n",
      "epoch:11,loss:0.3598\n",
      "0.895703125\n",
      "epoch:11,loss:0.2690\n",
      "0.8953125\n",
      "epoch:11,loss:0.2790\n",
      "0.8954427083333333\n",
      "epoch:11,loss:0.2431\n",
      "0.8955150462962963\n",
      "epoch:11,loss:0.4161\n",
      "0.8958333333333334\n",
      "epoch:11,loss:0.1826\n",
      "0.8960700757575758\n",
      "epoch:11,loss:0.2588\n",
      "0.8965060763888889\n",
      "epoch:11,loss:0.2482\n",
      "0.896053685897436\n",
      "============================================\n",
      "准确率由： 0.8748625942211056 上升至： 0.8959249371859297 已更新并保存权值为weights/best.pkl\n",
      "第11个epoch的识别准确率为：89%\n",
      "epoch:12,loss:0.2548\n",
      "0.9197916666666667\n",
      "epoch:12,loss:0.3332\n",
      "0.911328125\n",
      "epoch:12,loss:0.2013\n",
      "0.9119791666666667\n",
      "epoch:12,loss:0.2408\n",
      "0.9126953125\n",
      "epoch:12,loss:0.2736\n",
      "0.9119270833333334\n",
      "epoch:12,loss:0.2108\n",
      "0.9123697916666667\n",
      "epoch:12,loss:0.2632\n",
      "0.9113467261904762\n",
      "epoch:12,loss:0.2178\n",
      "0.91181640625\n",
      "epoch:12,loss:0.2965\n",
      "0.9125578703703704\n",
      "epoch:12,loss:0.2214\n",
      "0.9116927083333334\n",
      "epoch:12,loss:0.2949\n",
      "0.9104403409090909\n",
      "epoch:12,loss:0.1312\n",
      "0.910546875\n",
      "epoch:12,loss:0.1906\n",
      "0.9106971153846154\n",
      "============================================\n",
      "准确率由： 0.8959249371859297 上升至： 0.9104703203517588 已更新并保存权值为weights/best.pkl\n",
      "第12个epoch的识别准确率为：91%\n",
      "epoch:13,loss:0.2520\n",
      "0.9255208333333333\n",
      "epoch:13,loss:0.1921\n",
      "0.928125\n",
      "epoch:13,loss:0.2051\n",
      "0.9265625\n",
      "epoch:13,loss:0.1982\n",
      "0.9264973958333333\n",
      "epoch:13,loss:0.2101\n",
      "0.9258854166666667\n",
      "epoch:13,loss:0.1682\n",
      "0.9250868055555556\n",
      "epoch:13,loss:0.1890\n",
      "0.92578125\n",
      "epoch:13,loss:0.2624\n",
      "0.9254231770833333\n",
      "epoch:13,loss:0.1828\n",
      "0.9254918981481481\n",
      "epoch:13,loss:0.2020\n",
      "0.9248697916666667\n",
      "epoch:13,loss:0.3326\n",
      "0.9240767045454545\n",
      "epoch:13,loss:0.1024\n",
      "0.9234809027777777\n",
      "epoch:13,loss:0.1475\n",
      "0.9226161858974359\n",
      "============================================\n",
      "准确率由： 0.9104703203517588 上升至： 0.9223657349246231 已更新并保存权值为weights/best.pkl\n",
      "第13个epoch的识别准确率为：92%\n",
      "epoch:14,loss:0.1778\n",
      "0.93984375\n",
      "epoch:14,loss:0.1572\n",
      "0.934375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14,loss:0.1058\n",
      "0.9334201388888889\n",
      "epoch:14,loss:0.2383\n",
      "0.9317708333333333\n",
      "epoch:14,loss:0.1193\n",
      "0.9315104166666667\n",
      "epoch:14,loss:0.1947\n",
      "0.93125\n",
      "epoch:14,loss:0.2053\n",
      "0.9319196428571429\n",
      "epoch:14,loss:0.2591\n",
      "0.9318684895833333\n",
      "epoch:14,loss:0.1747\n",
      "0.9319155092592593\n",
      "epoch:14,loss:0.1940\n",
      "0.9318489583333334\n",
      "epoch:14,loss:0.1960\n",
      "0.9315104166666667\n",
      "epoch:14,loss:0.1874\n",
      "0.9319010416666667\n",
      "epoch:14,loss:0.2031\n",
      "0.9326522435897436\n",
      "============================================\n",
      "准确率由： 0.9223657349246231 上升至： 0.9325730213567839 已更新并保存权值为weights/best.pkl\n",
      "第14个epoch的识别准确率为：93%\n",
      "epoch:15,loss:0.2012\n",
      "0.9473958333333333\n",
      "epoch:15,loss:0.1051\n",
      "0.9404947916666667\n",
      "epoch:15,loss:0.1613\n",
      "0.9394965277777778\n",
      "epoch:15,loss:0.1573\n",
      "0.940234375\n",
      "epoch:15,loss:0.1967\n",
      "0.9399479166666667\n",
      "epoch:15,loss:0.1560\n",
      "0.9392795138888889\n",
      "epoch:15,loss:0.2908\n",
      "0.9389508928571428\n",
      "epoch:15,loss:0.1480\n",
      "0.9394205729166667\n",
      "epoch:15,loss:0.1854\n",
      "0.9394675925925926\n",
      "epoch:15,loss:0.2768\n",
      "0.939765625\n",
      "epoch:15,loss:0.1231\n",
      "0.9393939393939394\n",
      "epoch:15,loss:0.0922\n",
      "0.9392795138888889\n",
      "epoch:15,loss:0.2310\n",
      "0.9396233974358974\n",
      "============================================\n",
      "准确率由： 0.9325730213567839 上升至： 0.9396003454773869 已更新并保存权值为weights/best.pkl\n",
      "第15个epoch的识别准确率为：93%\n",
      "epoch:16,loss:0.1753\n",
      "0.9421875\n",
      "epoch:16,loss:0.0858\n",
      "0.94765625\n",
      "epoch:16,loss:0.1401\n",
      "0.9477430555555556\n",
      "epoch:16,loss:0.1077\n",
      "0.9481770833333333\n",
      "epoch:16,loss:0.1237\n",
      "0.9478645833333333\n",
      "epoch:16,loss:0.1127\n",
      "0.9476128472222223\n",
      "epoch:16,loss:0.1486\n",
      "0.9472098214285715\n",
      "epoch:16,loss:0.1665\n",
      "0.946875\n",
      "epoch:16,loss:0.1231\n",
      "0.9466145833333334\n",
      "epoch:16,loss:0.0793\n",
      "0.9465104166666667\n",
      "epoch:16,loss:0.1390\n",
      "0.9463304924242424\n",
      "epoch:16,loss:0.1422\n",
      "0.9451822916666667\n",
      "epoch:16,loss:0.1236\n",
      "0.9447315705128205\n",
      "============================================\n",
      "准确率由： 0.9396003454773869 上升至： 0.9446843592964824 已更新并保存权值为weights/best.pkl\n",
      "第16个epoch的识别准确率为：94%\n",
      "epoch:17,loss:0.1560\n",
      "0.9447916666666667\n",
      "epoch:17,loss:0.0969\n",
      "0.95\n",
      "epoch:17,loss:0.0833\n",
      "0.9515625\n",
      "epoch:17,loss:0.1642\n",
      "0.9502604166666667\n",
      "epoch:17,loss:0.1398\n",
      "0.9494791666666667\n",
      "epoch:17,loss:0.1318\n",
      "0.9503472222222222\n",
      "epoch:17,loss:0.0699\n",
      "0.9507440476190476\n",
      "epoch:17,loss:0.0894\n",
      "0.95068359375\n",
      "epoch:17,loss:0.1313\n",
      "0.9510127314814815\n",
      "epoch:17,loss:0.0870\n",
      "0.951171875\n",
      "epoch:17,loss:0.1837\n",
      "0.9509232954545455\n",
      "epoch:17,loss:0.1057\n",
      "0.9502387152777778\n",
      "epoch:17,loss:0.2399\n",
      "0.9497796474358975\n",
      "============================================\n",
      "准确率由： 0.9446843592964824 上升至： 0.9495720791457286 已更新并保存权值为weights/best.pkl\n",
      "第17个epoch的识别准确率为：94%\n",
      "epoch:18,loss:0.0399\n",
      "0.9645833333333333\n",
      "epoch:18,loss:0.1213\n",
      "0.9604166666666667\n",
      "epoch:18,loss:0.1769\n",
      "0.9605902777777777\n",
      "epoch:18,loss:0.1292\n",
      "0.9598958333333333\n",
      "epoch:18,loss:0.2434\n",
      "0.9583854166666667\n",
      "epoch:18,loss:0.0813\n",
      "0.9586371527777777\n",
      "epoch:18,loss:0.1681\n",
      "0.9580729166666667\n",
      "epoch:18,loss:0.0751\n",
      "0.9569010416666667\n",
      "epoch:18,loss:0.1350\n",
      "0.9561631944444444\n",
      "epoch:18,loss:0.1868\n",
      "0.9558854166666667\n",
      "epoch:18,loss:0.1192\n",
      "0.955752840909091\n",
      "epoch:18,loss:0.2038\n",
      "0.9551215277777778\n",
      "epoch:18,loss:0.1288\n",
      "0.9545873397435898\n",
      "============================================\n",
      "准确率由： 0.9495720791457286 上升至： 0.9544990577889447 已更新并保存权值为weights/best.pkl\n",
      "第18个epoch的识别准确率为：95%\n",
      "epoch:19,loss:0.0824\n",
      "0.9666666666666667\n",
      "epoch:19,loss:0.0707\n",
      "0.964453125\n",
      "epoch:19,loss:0.0537\n",
      "0.9634548611111111\n",
      "epoch:19,loss:0.0609\n",
      "0.9610026041666667\n",
      "epoch:19,loss:0.1095\n",
      "0.9602083333333333\n",
      "epoch:19,loss:0.1008\n",
      "0.9601996527777777\n",
      "epoch:19,loss:0.1273\n",
      "0.9595982142857142\n",
      "epoch:19,loss:0.0731\n",
      "0.9593098958333334\n",
      "epoch:19,loss:0.0960\n",
      "0.9580150462962963\n",
      "epoch:19,loss:0.1190\n",
      "0.9581510416666666\n",
      "epoch:19,loss:0.1864\n",
      "0.958001893939394\n",
      "epoch:19,loss:0.0958\n",
      "0.9579427083333333\n",
      "epoch:19,loss:0.0452\n",
      "0.9579126602564103\n",
      "============================================\n",
      "准确率由： 0.9544990577889447 上升至： 0.9579538316582915 已更新并保存权值为weights/best.pkl\n",
      "第19个epoch的识别准确率为：95%\n",
      "epoch:20,loss:0.2198\n",
      "0.9627604166666667\n",
      "epoch:20,loss:0.0610\n",
      "0.9631510416666667\n",
      "epoch:20,loss:0.0700\n",
      "0.96171875\n",
      "epoch:20,loss:0.1375\n",
      "0.9606770833333333\n",
      "epoch:20,loss:0.1515\n",
      "0.9602083333333333\n",
      "epoch:20,loss:0.0732\n",
      "0.9618055555555556\n",
      "epoch:20,loss:0.1147\n",
      "0.962313988095238\n",
      "epoch:20,loss:0.0758\n",
      "0.96220703125\n",
      "epoch:20,loss:0.1880\n",
      "0.961400462962963\n",
      "epoch:20,loss:0.1613\n",
      "0.96125\n",
      "epoch:20,loss:0.0879\n",
      "0.9605350378787879\n",
      "epoch:20,loss:0.1409\n",
      "0.9605685763888889\n",
      "epoch:20,loss:0.0751\n",
      "0.9606370192307693\n",
      "============================================\n",
      "准确率由： 0.9579538316582915 上升至： 0.9603093592964824 已更新并保存权值为weights/best.pkl\n",
      "第20个epoch的识别准确率为：96%\n",
      "epoch:21,loss:0.0895\n",
      "0.95625\n",
      "epoch:21,loss:0.0880\n",
      "0.9625\n",
      "epoch:21,loss:0.1067\n",
      "0.9650173611111111\n",
      "epoch:21,loss:0.0346\n",
      "0.9661458333333334\n",
      "epoch:21,loss:0.2700\n",
      "0.9656770833333334\n",
      "epoch:21,loss:0.0846\n",
      "0.9655381944444444\n",
      "epoch:21,loss:0.0834\n",
      "0.9643229166666667\n",
      "epoch:21,loss:0.0369\n",
      "0.9646809895833334\n",
      "epoch:21,loss:0.0523\n",
      "0.9641782407407408\n",
      "epoch:21,loss:0.0884\n",
      "0.96421875\n",
      "epoch:21,loss:0.1140\n",
      "0.9641571969696969\n",
      "epoch:21,loss:0.1848\n",
      "0.9633680555555556\n",
      "epoch:21,loss:0.0770\n",
      "0.9627804487179488\n",
      "============================================\n",
      "准确率由： 0.9603093592964824 上升至： 0.9625667399497487 已更新并保存权值为weights/best.pkl\n",
      "第21个epoch的识别准确率为：96%\n",
      "epoch:22,loss:0.0400\n",
      "0.9669270833333333\n",
      "epoch:22,loss:0.1504\n",
      "0.9704427083333333\n",
      "epoch:22,loss:0.0804\n",
      "0.97109375\n",
      "epoch:22,loss:0.0638\n",
      "0.9692057291666667\n",
      "epoch:22,loss:0.1174\n",
      "0.9672395833333334\n",
      "epoch:22,loss:0.0940\n",
      "0.9661892361111111\n",
      "epoch:22,loss:0.0651\n",
      "0.9660714285714286\n",
      "epoch:22,loss:0.1212\n",
      "0.9661783854166667\n",
      "epoch:22,loss:0.0768\n",
      "0.9666956018518519\n",
      "epoch:22,loss:0.0533\n",
      "0.9664583333333333\n",
      "epoch:22,loss:0.1050\n",
      "0.9662405303030303\n",
      "epoch:22,loss:0.1517\n",
      "0.9655164930555555\n",
      "epoch:22,loss:0.0557\n",
      "0.9651842948717949\n",
      "============================================\n",
      "准确率由： 0.9625667399497487 上升至： 0.9652952261306532 已更新并保存权值为weights/best.pkl\n",
      "第22个epoch的识别准确率为：96%\n",
      "epoch:23,loss:0.0410\n",
      "0.978125\n",
      "epoch:23,loss:0.0705\n",
      "0.9725260416666667\n",
      "epoch:23,loss:0.1127\n",
      "0.9691840277777778\n",
      "epoch:23,loss:0.0729\n",
      "0.9685546875\n",
      "epoch:23,loss:0.1097\n",
      "0.96859375\n",
      "epoch:23,loss:0.1068\n",
      "0.96875\n",
      "epoch:23,loss:0.0798\n",
      "0.9689360119047619\n",
      "epoch:23,loss:0.0733\n",
      "0.9686197916666667\n",
      "epoch:23,loss:0.1811\n",
      "0.9678819444444444\n",
      "epoch:23,loss:0.0534\n",
      "0.9672916666666667\n",
      "epoch:23,loss:0.0983\n",
      "0.9675189393939394\n",
      "epoch:23,loss:0.0538\n",
      "0.9676215277777778\n",
      "epoch:23,loss:0.1207\n",
      "0.9673677884615385\n",
      "============================================\n",
      "准确率由： 0.9652952261306532 上升至： 0.9672974246231156 已更新并保存权值为weights/best.pkl\n",
      "第23个epoch的识别准确率为：96%\n",
      "epoch:24,loss:0.1033\n",
      "0.9723958333333333\n",
      "epoch:24,loss:0.0538\n",
      "0.9735677083333333\n",
      "epoch:24,loss:0.1846\n",
      "0.975\n",
      "epoch:24,loss:0.0618\n",
      "0.9743489583333333\n",
      "epoch:24,loss:0.0930\n",
      "0.9746354166666666\n",
      "epoch:24,loss:0.0680\n",
      "0.97421875\n",
      "epoch:24,loss:0.0508\n",
      "0.973735119047619\n",
      "epoch:24,loss:0.0975\n",
      "0.9735026041666667\n",
      "epoch:24,loss:0.0907\n",
      "0.97265625\n",
      "epoch:24,loss:0.1742\n",
      "0.9714583333333333\n",
      "epoch:24,loss:0.0720\n",
      "0.9711410984848485\n",
      "epoch:24,loss:0.0246\n",
      "0.9707682291666667\n",
      "epoch:24,loss:0.0807\n",
      "0.9697916666666667\n",
      "============================================\n",
      "准确率由： 0.9672974246231156 上升至： 0.9698885050251256 已更新并保存权值为weights/best.pkl\n",
      "第24个epoch的识别准确率为：96%\n",
      "epoch:25,loss:0.0793\n",
      "0.98046875\n",
      "epoch:25,loss:0.0827\n",
      "0.978125\n",
      "epoch:25,loss:0.0439\n",
      "0.9775173611111111\n",
      "epoch:25,loss:0.1869\n",
      "0.9758463541666667\n",
      "epoch:25,loss:0.1092\n",
      "0.9747395833333333\n",
      "epoch:25,loss:0.0763\n",
      "0.9743055555555555\n",
      "epoch:25,loss:0.0653\n",
      "0.9740699404761904\n",
      "epoch:25,loss:0.0708\n",
      "0.9732096354166667\n",
      "epoch:25,loss:0.0871\n",
      "0.9718171296296296\n",
      "epoch:25,loss:0.0970\n",
      "0.9713802083333334\n",
      "epoch:25,loss:0.2016\n",
      "0.970999053030303\n",
      "epoch:25,loss:0.0723\n",
      "0.9711588541666667\n",
      "epoch:25,loss:0.0664\n",
      "0.9712940705128205\n",
      "============================================\n",
      "准确率由： 0.9698885050251256 上升至： 0.9712821922110553 已更新并保存权值为weights/best.pkl\n",
      "第25个epoch的识别准确率为：97%\n",
      "epoch:26,loss:0.0593\n",
      "0.97890625\n",
      "epoch:26,loss:0.0571\n",
      "0.9751302083333333\n",
      "epoch:26,loss:0.0720\n",
      "0.9730034722222223\n",
      "epoch:26,loss:0.0331\n",
      "0.971484375\n",
      "epoch:26,loss:0.0289\n",
      "0.971875\n",
      "epoch:26,loss:0.0810\n",
      "0.9730034722222223\n",
      "epoch:26,loss:0.0334\n",
      "0.9726190476190476\n",
      "epoch:26,loss:0.0848\n",
      "0.97265625\n",
      "epoch:26,loss:0.0927\n",
      "0.9723090277777777\n",
      "epoch:26,loss:0.0503\n",
      "0.9728645833333334\n",
      "epoch:26,loss:0.0746\n",
      "0.9730823863636363\n",
      "epoch:26,loss:0.0115\n",
      "0.9733940972222223\n",
      "epoch:26,loss:0.0236\n",
      "0.9737179487179487\n",
      "============================================\n",
      "准确率由： 0.9712821922110553 上升至： 0.9736573492462312 已更新并保存权值为weights/best.pkl\n",
      "第26个epoch的识别准确率为：97%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:27,loss:0.1044\n",
      "0.97421875\n",
      "epoch:27,loss:0.0507\n",
      "0.9723958333333333\n",
      "epoch:27,loss:0.0340\n",
      "0.9716145833333333\n",
      "epoch:27,loss:0.0730\n",
      "0.97109375\n",
      "epoch:27,loss:0.1546\n",
      "0.9724479166666666\n",
      "epoch:27,loss:0.0401\n",
      "0.9723958333333333\n",
      "epoch:27,loss:0.0283\n",
      "0.9728794642857143\n",
      "epoch:27,loss:0.0684\n",
      "0.9730794270833333\n",
      "epoch:27,loss:0.0190\n",
      "0.9730613425925926\n",
      "epoch:27,loss:0.0839\n",
      "0.97234375\n",
      "epoch:27,loss:0.0774\n",
      "0.9726799242424242\n",
      "epoch:27,loss:0.0437\n",
      "0.9726128472222222\n",
      "epoch:27,loss:0.0484\n",
      "0.9726762820512821\n",
      "============================================\n",
      "第27个epoch的识别准确率为：97%\n",
      "epoch:28,loss:0.0345\n",
      "0.9791666666666666\n",
      "epoch:28,loss:0.0862\n",
      "0.9774739583333333\n",
      "epoch:28,loss:0.0326\n",
      "0.9789930555555556\n",
      "epoch:28,loss:0.0242\n",
      "0.9778645833333334\n",
      "epoch:28,loss:0.1033\n",
      "0.9771354166666667\n",
      "epoch:28,loss:0.1142\n",
      "0.9763454861111112\n",
      "epoch:28,loss:0.1398\n",
      "0.9757068452380953\n",
      "epoch:28,loss:0.0494\n",
      "0.9750325520833333\n",
      "epoch:28,loss:0.0357\n",
      "0.9754918981481482\n",
      "epoch:28,loss:0.0628\n",
      "0.9753645833333333\n",
      "epoch:28,loss:0.0652\n",
      "0.9755445075757576\n",
      "epoch:28,loss:0.1433\n",
      "0.9754340277777778\n",
      "epoch:28,loss:0.1111\n",
      "0.9754407051282051\n",
      "============================================\n",
      "准确率由： 0.9736573492462312 上升至： 0.9755025125628141 已更新并保存权值为weights/best.pkl\n",
      "第28个epoch的识别准确率为：97%\n",
      "epoch:29,loss:0.0771\n",
      "0.9809895833333333\n",
      "epoch:29,loss:0.0148\n",
      "0.9817708333333334\n",
      "epoch:29,loss:0.0367\n",
      "0.9809027777777778\n",
      "epoch:29,loss:0.0212\n",
      "0.9797526041666667\n",
      "epoch:29,loss:0.0495\n",
      "0.9795833333333334\n",
      "epoch:29,loss:0.0360\n",
      "0.9788194444444445\n",
      "epoch:29,loss:0.0639\n",
      "0.9784226190476191\n",
      "epoch:29,loss:0.0278\n",
      "0.9784505208333333\n",
      "epoch:29,loss:0.0711\n",
      "0.9784432870370371\n",
      "epoch:29,loss:0.1146\n",
      "0.978046875\n",
      "epoch:29,loss:0.0104\n",
      "0.9779592803030303\n",
      "epoch:29,loss:0.0907\n",
      "0.9773220486111112\n",
      "epoch:29,loss:0.0327\n",
      "0.9769831730769231\n",
      "============================================\n",
      "准确率由： 0.9755025125628141 上升至： 0.9768765703517588 已更新并保存权值为weights/best.pkl\n",
      "第29个epoch的识别准确率为：97%\n",
      "epoch:30,loss:0.0501\n",
      "0.9786458333333333\n",
      "epoch:30,loss:0.0465\n",
      "0.978515625\n",
      "epoch:30,loss:0.0462\n",
      "0.9802083333333333\n",
      "epoch:30,loss:0.1104\n",
      "0.9795572916666667\n",
      "epoch:30,loss:0.0666\n",
      "0.9789583333333334\n",
      "epoch:30,loss:0.0641\n",
      "0.9787326388888888\n",
      "epoch:30,loss:0.0863\n",
      "0.9783482142857143\n",
      "epoch:30,loss:0.0611\n",
      "0.9785807291666667\n",
      "epoch:30,loss:0.0452\n",
      "0.9786168981481481\n",
      "epoch:30,loss:0.0769\n",
      "0.9785416666666666\n",
      "epoch:30,loss:0.0367\n",
      "0.9789299242424242\n",
      "epoch:30,loss:0.0215\n",
      "0.9786241319444444\n",
      "epoch:30,loss:0.0893\n",
      "0.9788261217948718\n",
      "============================================\n",
      "准确率由： 0.9768765703517588 上升至： 0.9788002512562815 已更新并保存权值为weights/best.pkl\n",
      "第30个epoch的识别准确率为：97%\n",
      "epoch:31,loss:0.0119\n",
      "0.9799479166666667\n",
      "epoch:31,loss:0.1064\n",
      "0.9787760416666667\n",
      "epoch:31,loss:0.0520\n",
      "0.9764756944444445\n",
      "epoch:31,loss:0.0256\n",
      "0.977734375\n",
      "epoch:31,loss:0.1163\n",
      "0.97890625\n",
      "epoch:31,loss:0.0369\n",
      "0.9787760416666667\n",
      "epoch:31,loss:0.0321\n",
      "0.9787946428571429\n",
      "epoch:31,loss:0.0252\n",
      "0.9787434895833333\n",
      "epoch:31,loss:0.0520\n",
      "0.9779513888888889\n",
      "epoch:31,loss:0.1388\n",
      "0.976796875\n",
      "epoch:31,loss:0.1173\n",
      "0.9769176136363636\n",
      "epoch:31,loss:0.1770\n",
      "0.9769097222222223\n",
      "epoch:31,loss:0.0352\n",
      "0.9767427884615385\n",
      "============================================\n",
      "第31个epoch的识别准确率为：97%\n",
      "epoch:32,loss:0.0834\n",
      "0.9796875\n",
      "epoch:32,loss:0.1946\n",
      "0.9799479166666667\n",
      "epoch:32,loss:0.1031\n",
      "0.9808159722222223\n",
      "epoch:32,loss:0.0629\n",
      "0.9810546875\n",
      "epoch:32,loss:0.0705\n",
      "0.9815104166666667\n",
      "epoch:32,loss:0.0525\n",
      "0.9816840277777777\n",
      "epoch:32,loss:0.0672\n",
      "0.9809523809523809\n",
      "epoch:32,loss:0.0251\n",
      "0.98076171875\n",
      "epoch:32,loss:0.1139\n",
      "0.9801793981481481\n",
      "epoch:32,loss:0.0810\n",
      "0.9797395833333333\n",
      "epoch:32,loss:0.0907\n",
      "0.9788825757575758\n",
      "epoch:32,loss:0.0191\n",
      "0.9784939236111111\n",
      "epoch:32,loss:0.0342\n",
      "0.9783052884615384\n",
      "============================================\n",
      "第32个epoch的识别准确率为：97%\n",
      "epoch:33,loss:0.0268\n",
      "0.9846354166666667\n",
      "epoch:33,loss:0.0401\n",
      "0.982421875\n",
      "epoch:33,loss:0.0780\n",
      "0.9821180555555555\n",
      "epoch:33,loss:0.0215\n",
      "0.9826822916666667\n",
      "epoch:33,loss:0.0905\n",
      "0.9816666666666667\n",
      "epoch:33,loss:0.1475\n",
      "0.9810763888888889\n"
     ]
    }
   ],
   "source": [
    "# ============================ step 5/5 训练 ============================\n",
    "# 记录每一次的数据，方便绘图\n",
    "train_curve=list()\n",
    "valid_curve=list()\n",
    "model.train()\n",
    "accurancy_global=0.0\n",
    "# model.layers[-1].register_forward_hook(farward_hook)\n",
    "# model.layers[-1].register_backward_hook(backward_hook)\n",
    "# model.layers[-5].register_backward_hook(backward2_hook)\n",
    "# model.layers[-11].register_backward_hook(backward3_hook)\n",
    "for epoch in range(MAX_EPOCH):\n",
    "    loss_mean=0.\n",
    "    correct=0.\n",
    "    total=0.\n",
    "    running_loss = 0.0\n",
    "\n",
    "    for i,data in enumerate(train_loader):\n",
    "        \n",
    "        img,label=data\n",
    "        img = Variable(img)\n",
    "        label = Variable(label)\n",
    "        if torch.cuda.is_available():\n",
    "            img=img.cuda()\n",
    "            label=label.cuda()\n",
    "        \n",
    "        out=model(img)\n",
    "        \n",
    "#         optimizer2.zero_grad()\n",
    "#         class_loss=gs(out,label)\n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        \n",
    "#         w_f,w_m,w_l=count_final(grad_block,grad_block2,grad_block3,BATCH_SIZE)\n",
    "        \n",
    "            \n",
    "#         grad_block=[]\n",
    "#         grad_block2=[]\n",
    "#         grad_block3=[]\n",
    "        \n",
    "        optimizer.zero_grad()  # 归0梯度\n",
    "        ###########################################\n",
    "#         print_loss=loss.data.item()\n",
    "        ###########################################\n",
    "        \n",
    "        loss=criterion(out,label)#得到损失函数\n",
    "        \n",
    "#         class_loss.backward(retain_graph=True)\n",
    "        ###########################################\n",
    "        loss.backward(retain_graph=False)#反向传播\n",
    "        ###########################################\n",
    "        optimizer.step()#优化\n",
    "        \n",
    "        \n",
    "        if (i+1)%log_interval==0:\n",
    "            print('epoch:{},loss:{:.4f}'.format(epoch+1,loss.data.item()))\n",
    "        _, predicted = torch.max(out.data, 1)\n",
    "        total += label.size(0)\n",
    "#         if (i+1)%log_interval==0:\n",
    "#             print(\"============================================\")\n",
    "#             print(\"源数据标签：\",label)\n",
    "#             print(\"============================================\")\n",
    "#             print(\"预测结果：\",predicted)\n",
    "#             print(\"相等的结果为：\",predicted == label)\n",
    "        correct += (predicted == label).sum()\n",
    "        if (i+1)%log_interval==0:\n",
    "            print(correct.item() / total)\n",
    "#         print(correct.item())\n",
    "\n",
    "#     fi=epoch\n",
    "#     if (w_l>0.7)and(fi==0):\n",
    "#         fi+=1\n",
    "#         model.weights_try=w_f\n",
    "#         model.weights2_try=w_m\n",
    "#         model.weights3_try=w_l\n",
    "#         print(w_f,w_m,w_l)\n",
    "#         print('固定参数')\n",
    "#         print(fi)\n",
    "#     model.weights_try=w_f\n",
    "#     model.weights2_try=w_m\n",
    "#     model.weights3_try=w_l\n",
    "    print(\"============================================\")\n",
    "#     print(w_f,w_m,w_l)\n",
    "    accurancy=correct.item() / total\n",
    "    if accurancy>accurancy_global:\n",
    "        torch.save(model.state_dict(), './weights/bestcat.pkl')\n",
    "        print(\"准确率由：\", accurancy_global, \"上升至：\", accurancy, \"已更新并保存权值为weights/best.pkl\")\n",
    "        accurancy_global=accurancy\n",
    "    print('第%d个epoch的识别准确率为：%d%%' % (epoch + 1, 100*accurancy))\n",
    "# torch.save(model.state_dict(), './weights/last322.pkl')\n",
    "print(\"训练完毕，权重已保存为：weights/last.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
